{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.nn import init\n",
    "import math\n",
    "import os\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ChannelAttention(nn.Module):\n",
    "    def __init__(self, in_planes, ratio=16):\n",
    "        super(ChannelAttention, self).__init__()\n",
    "        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n",
    "        self.max_pool = nn.AdaptiveMaxPool2d(1)\n",
    "        self.fc1   = nn.Conv2d(in_planes, in_planes // 16, 1, bias=False)\n",
    "        self.relu1 = nn.ReLU()\n",
    "        self.fc2   = nn.Conv2d(in_planes // 16, in_planes, 1, bias=False)\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "\n",
    "    def forward(self, x):\n",
    "        avg_out = self.fc2(self.relu1(self.fc1(self.avg_pool(x))))\n",
    "        max_out = self.fc2(self.relu1(self.fc1(self.max_pool(x))))\n",
    "        out = avg_out + max_out\t\t# 这里并没有使用到论文中的shared MLP, 而是直接相加了\n",
    "        return self.sigmoid(out)\n",
    "\n",
    "\n",
    "class SpatialAttention(nn.Module):\n",
    "    def __init__(self, kernel_size=7):\n",
    "        super(SpatialAttention, self).__init__()\n",
    "        assert kernel_size in (3, 7), 'kernel size must be 3 or 7'\n",
    "        padding = 3 if kernel_size == 7 else 1\n",
    "        self.conv1 = nn.Conv2d(2, 1, kernel_size, padding=padding, bias=False)\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "\n",
    "    def forward(self, x):\n",
    "        avg_out = torch.mean(x, dim=1, keepdim=True)\n",
    "        max_out, _ = torch.max(x, dim=1, keepdim=True)\n",
    "        x = torch.cat([avg_out, max_out], dim=1)\n",
    "        x = self.conv1(x)\n",
    "        return self.sigmoid(x)\n",
    "def conv3x3(in_planes, out_planes, stride=1):\n",
    "    \"3x3 convolution with padding\"\n",
    "    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n",
    "                     padding=1, bias=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SEModule(nn.Module):\n",
    "    expansion = 1\n",
    "\n",
    "    def __init__(self, inplanes, stride=1, downsample=None):\n",
    "        super(SEModule, self).__init__()\n",
    "        self.conv1 = conv3x3(inplanes, inplanes, stride)\n",
    "        self.bn1 = nn.BatchNorm2d(inplanes)\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.conv2 = conv3x3(inplanes, inplanes)\n",
    "        self.bn2 = nn.BatchNorm2d(inplanes)\n",
    "\n",
    "        self.ca = ChannelAttention(inplanes)\n",
    "        self.sa = SpatialAttention()\n",
    "\n",
    "        self.downsample = downsample\n",
    "        self.stride = stride\n",
    "\n",
    "    def forward(self, x):\n",
    "        residual = x\n",
    "\n",
    "        out = self.conv1(x)\n",
    "        out = self.bn1(out)\n",
    "        out = self.relu(out)\n",
    "\n",
    "        out = self.conv2(out)\n",
    "        out = self.bn2(out)\n",
    "\n",
    "        out = self.ca(out) * out\n",
    "       \n",
    "        out += residual\n",
    "      \n",
    "#         print(out.size())\n",
    "        out = self.sa(out) * out\n",
    "#         print(out.size())\n",
    "        if self.downsample is not None:\n",
    "            residual = self.downsample(x)\n",
    "\n",
    "        out += residual\n",
    "#         print(out.size())\n",
    "        out = self.relu(out)\n",
    "\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv31(ip,op,stride=1):\n",
    "    return nn.Conv2d(ip,op,kernel_size=3,stride=stride,padding=1,bias=False)\n",
    "def conv32(ip,op,stride=2):\n",
    "    return nn.Conv2d(ip,op,kernel_size=3,stride=stride,padding=1,bias=False)\n",
    "class Upsample(nn.Module):\n",
    "    def __init__(self,ip,op):\n",
    "        super(Upsample,self).__init__()\n",
    "        self.upsample=nn.Sequential(\n",
    "            conv31(ip,op,1),\n",
    "            nn.Upsample(scale_factor=2,mode='nearest')\n",
    "        )\n",
    "        \n",
    "    def forward(self,x):\n",
    "        x=self.upsample(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def backward_hook(module,grad_in,grad_out):\n",
    "#     grad_block.append(grad_out[0].detach())\n",
    "\n",
    "# def farward_hook(module,input,output):\n",
    "#     fmap_block.append(output)\n",
    "\n",
    "    \n",
    "# def backward2_hook(module,grad_in,grad_out):\n",
    "#     grad_block2.append(grad_out[0].detach())\n",
    "# def backward3_hook(module,grad_in,grad_out):\n",
    "#     grad_block3.append(grad_out[0].detach())\n",
    "# fmap_block=[]\n",
    "# grad_block=[]\n",
    "# grad_block2=[]\n",
    "# grad_block3=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Hswish(x,inplace=True):\n",
    "    return x * F.relu6(x + 3., inplace=inplace) / 6.\n",
    "\n",
    "def Hsigmoid(x,inplace=True):\n",
    "    return F.relu6(x + 3., inplace=inplace) / 6.\n",
    "\n",
    "\n",
    "class Bottleneck(nn.Module):\n",
    "    def __init__(self,in_channels,out_channels,kernel_size,exp_channels,stride,se='True',nl='HS'):\n",
    "        super(Bottleneck, self).__init__()\n",
    "        padding = (kernel_size - 1) // 2\n",
    "        if nl == 'RE':\n",
    "            self.nlin_layer = F.relu6\n",
    "        elif nl == 'HS':\n",
    "            self.nlin_layer = Hswish\n",
    "        self.stride=stride\n",
    "        if se:\n",
    "            self.se=SEModule(exp_channels)\n",
    "        else:\n",
    "            self.se=None\n",
    "        self.conv1=nn.Conv2d(in_channels,exp_channels,kernel_size=1,stride=1,padding=0,bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(exp_channels)\n",
    "        self.conv2=nn.Conv2d(exp_channels,exp_channels,kernel_size=kernel_size,stride=stride,\n",
    "                             padding=padding,groups=exp_channels,bias=False)\n",
    "        self.bn2=nn.BatchNorm2d(exp_channels)\n",
    "        self.conv3=nn.Conv2d(exp_channels,out_channels,kernel_size=1,stride=1,padding=0,bias=False)\n",
    "        self.bn3=nn.BatchNorm2d(out_channels)\n",
    "        # ÏÈ³õÊ¼»¯Ò»¸ö¿ÕÐòÁÐ£¬Ö®ºó¸ÄÔìÆä³ÉÎª²Ð²îÁ´½Ó\n",
    "        self.shortcut = nn.Sequential()\n",
    "        # Ö»ÓÐ²½³¤Îª1ÇÒÊäÈëÊä³öÍ¨µÀ²»ÏàÍ¬Ê±²Å²ÉÓÃÌøÔ¾Á¬½Ó(ÏëÒ»ÏÂÌøÔ¾Á´½ÓµÄ¹ý³Ì£¬ÊäÈëÊä³öÍ¨µÀÏàÍ¬Õâ¸öÌøÔ¾Á¬½Ó¾ÍÃ»ÒâÒåÁË)\n",
    "        if stride == 1 and in_channels != out_channels:\n",
    "            self.shortcut = nn.Sequential(\n",
    "                # ÏÂÃæµÄ²Ù×÷¾í»ý²»¸Ä±ä³ß´çºÍÍ¨µÀÊý\n",
    "                nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0, bias=False),\n",
    "                nn.BatchNorm2d(out_channels)\n",
    "            )\n",
    "\n",
    "    def forward(self,x):\n",
    "        out=self.nlin_layer(self.bn1(self.conv1(x)))\n",
    "        if self.se is not None:\n",
    "            out=self.bn2(self.conv2(out))\n",
    "            out=self.nlin_layer(self.se(out))\n",
    "        else:\n",
    "            out = self.nlin_layer(self.bn2(self.conv2(out)))\n",
    "        out = self.bn3(self.conv3(out))\n",
    "        out = out + self.shortcut(x) if self.stride == 1 else out\n",
    "        return out\n",
    "\n",
    "\n",
    "class MobileNetV3_large(nn.Module):\n",
    "    # (out_channels,kernel_size,exp_channels,stride,se,nl)\n",
    "    cfg=[\n",
    "        (16,3,16,1,False,'RE'),\n",
    "        (24,3,64,2,False,'RE'),\n",
    "        (24,3,72,1,False,'RE'),\n",
    "        (40,5,72,2,True,'RE'),\n",
    "        (40,5,120,1,True,'RE'),\n",
    "        (40,5,120,1,True,'RE'),\n",
    "        (80,3,240,2,False,'HS'),\n",
    "        (80,3,200,1,False,'HS'),\n",
    "        (80,3,184,1,False,'HS'),\n",
    "        (80,3,184,1,False,'HS'),\n",
    "        (112,3,480,1,True,'HS'),\n",
    "        (112,3,672,1,True,'HS'),\n",
    "        (160,5,672,2,True,'HS'),\n",
    "        (160,5,960,1,True,'HS'),\n",
    "        (160,5,960,1,True,'HS')\n",
    "    ]\n",
    "    def __init__(self,num_classes=5):\n",
    "        super(MobileNetV3_large,self).__init__()\n",
    "        self.conv1=nn.Conv2d(3,16,3,2,padding=1,bias=False)\n",
    "        self.bn1=nn.BatchNorm2d(16)\n",
    "        # ¸ù¾ÝcfgÊý×é×Ô¶¯Éú³ÉËùÓÐµÄBottleneck²ã\n",
    "        self.layers = self._make_layers(in_channels=16)\n",
    "        self.conv2=nn.Conv2d(160,960,1,stride=1,bias=False)\n",
    "        self.bn2=nn.BatchNorm2d(960)\n",
    "        # ¾í»ýºó²»¸úBN£¬¾ÍÓ¦¸Ã°ÑbiasÉèÖÃÎªTrue\n",
    "        self.conv3=nn.Conv2d(960,1280,1,1,padding=0,bias=True)\n",
    "        self.conv4=nn.Conv2d(1280,num_classes,1,stride=1,padding=0,bias=True)\n",
    "        self.up1=Upsample(160,112)\n",
    "        self.up2=Upsample(112,40)\n",
    "        self.c1=nn.Conv2d(224,112,kernel_size=3,stride=1,padding=1)\n",
    "        self.c2=nn.Conv2d(80,40,1,1)\n",
    "        self.c3=nn.Conv2d(40,112,kernel_size=3,stride=2,padding=1)\n",
    "        self.b4=nn.BatchNorm2d(112)\n",
    "        \n",
    "        self.c6=nn.Conv2d(112,160,kernel_size=3,stride=2,padding=1)\n",
    "        self.b7=nn.BatchNorm2d(160)\n",
    "        self.h1=nn.Conv2d(80,40,1,1)\n",
    "        self.h2=nn.Conv2d(224,112,1,1)\n",
    "        self.h3=nn.Conv2d(320,160,1,1)\n",
    "        \n",
    "#         self.w=nn.Parameter(torch.FloatTensor(1),requires_grad=True)\n",
    "#         self.w=torch.FloatTensor(1)\n",
    "        self.weights_try=0.1\n",
    "        self.weights2_try=0.2\n",
    "        self.weights3_try=0.7\n",
    "        \n",
    "#         self.w.data.fill_(1)\n",
    "#         self.weights2_try.data.fill_(1)\n",
    "#         self.weights3_try.data.fill_(1)\n",
    "        \n",
    "    def _make_layers(self,in_channels):\n",
    "        layers=[]\n",
    "        for out_channels,kernel_size,exp_channels,stride,se,nl in self.cfg:\n",
    "            layers.append(\n",
    "                Bottleneck(in_channels,out_channels,kernel_size,exp_channels,stride,se,nl)\n",
    "            )\n",
    "            in_channels=out_channels\n",
    "        return nn.Sequential(*layers)\n",
    "                \n",
    "\n",
    "    def forward(self,x):\n",
    "        out=Hswish(self.bn1(self.conv1(x)))\n",
    "        ##############################################\n",
    "        out1=self.layers[:6](out)\n",
    "        out2=self.layers[6:12](out1)\n",
    "        out3=self.layers[12:16](out2)\n",
    "        \n",
    "        out3x=self.weights3_try*out3\n",
    "        out1x=self.weights_try*out1\n",
    "        out2x=self.weights2_try*out2\n",
    "        \n",
    "#         out3=torch.cat([out3,out3x],1)\n",
    "#         out3=self.h3(out3)\n",
    "#         out2=torch.cat([out2,out2x],1)\n",
    "#         out2=self.h2(out2)\n",
    "#         out1=torch.cat([out1,out1x],1)\n",
    "#         out1=self.h1(out1)\n",
    "        p5=self.up1(out3)\n",
    "        p4=torch.cat([p5,out2],1)\n",
    "#         print(p4.size()) #14*14*224\n",
    "        p4=self.c1(p4)\n",
    "        p4=self.up2(p4)\n",
    "#         print(p4.size())\n",
    "        p3=torch.cat([p4,out1],1)\n",
    "        p3=self.c2(p3)\n",
    "        p3=self.c3(p3)\n",
    "        p3=self.b4(p3)\n",
    "        p3=Hswish(p3)\n",
    "        \n",
    "        p3=self.c6(p3)\n",
    "        p3=self.b7(p3)\n",
    "        p3=Hswish(p3)\n",
    " ##############################################       \n",
    "        p5y=self.up1(out3x)\n",
    "        p4=torch.cat([p5,out2],1)\n",
    "#         print(p4.size()) #14*14*224\n",
    "        p4=self.c1(p4)\n",
    "        p4=self.up2(p4)\n",
    "#         print(p4.size())\n",
    "        p3=torch.cat([p4,out1],1)\n",
    "        p3=self.c2(p3)\n",
    "        p3=self.c3(p3)\n",
    "        p3=self.b4(p3)\n",
    "        p3=Hswish(p3)\n",
    "        \n",
    "        p3=self.c6(p3)\n",
    "        p3=self.b7(p3)\n",
    "        p3=Hswish(p3)\n",
    "  #######################################################      \n",
    "        out=Hswish(self.bn2(self.conv2(p3)))\n",
    "        out=F.avg_pool2d(out,7)\n",
    "        out=Hswish(self.conv3(out))\n",
    "        out=self.conv4(out)\n",
    "        \n",
    "        a,b=out.size(0),out.size(1)\n",
    "        out=out.view(a,b)\n",
    "#         print(self.weights_try)\n",
    "#         return out,out1,out2,out3\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model=MobileNetV3_large()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x=torch.randn(2,3,224,224)\n",
    "# y=model(x)\n",
    "# print(y.size())\n",
    "# print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision\n",
    "from torch.autograd import Variable\n",
    "import torchvision.transforms as transforms\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "from PIL import Image\n",
    "from torch.utils.data import Dataset\n",
    "\n",
    "random.seed(1)\n",
    "\n",
    "class flowerDataset(Dataset):\n",
    "    # 自定义Dataset类，必须继承Dataset并重写__init__和__getitem__函数\n",
    "    def __init__(self, data_dir, transform=None):\n",
    "        \"\"\"\n",
    "            花朵分类任务的Dataset\n",
    "            :param data_dir: str, 数据集所在路径\n",
    "            :param transform: torch.transform，数据预处理，默认不进行预处理\n",
    "        \"\"\"\n",
    "        # data_info存储所有图片路径和标签（元组的列表），在DataLoader中通过index读取样本\n",
    "        self.data_info = self.get_img_info(data_dir)\n",
    "        self.transform = transform\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        path_img, label = self.data_info[index]\n",
    "        # 打开图片，默认为PIL，需要转成RGB\n",
    "        img = Image.open(path_img).convert('RGB')\n",
    "        # 如果预处理的条件不为空，应该进行预处理操作\n",
    "        if self.transform is not None:\n",
    "            img = self.transform(img)\n",
    "        return img, label\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.data_info)\n",
    "\n",
    "    # 自定义方法，用于返回所有图片的路径以及标签\n",
    "    @staticmethod\n",
    "    def get_img_info(data_dir):\n",
    "        data_info = list()\n",
    "        for root, dirs, _ in os.walk(data_dir):\n",
    "            # 遍历类别\n",
    "            for sub_dir in dirs:\n",
    "                # listdir为列出文件夹下所有文件和文件夹名\n",
    "                img_names = os.listdir(os.path.join(root, sub_dir))\n",
    "                # 过滤出所有后缀名为jpg的文件名（那当然也就把文件夹过滤掉了）\n",
    "                img_names = list(filter(lambda x: x.endswith('.png'), img_names))\n",
    "\n",
    "                # 遍历图片\n",
    "                for i in range(len(img_names)):\n",
    "                    img_name = img_names[i]\n",
    "                    path_img = os.path.join(root, sub_dir, img_name)\n",
    "                    # 在该任务中，文件夹名等于标签名\n",
    "                    label = sub_dir\n",
    "                    data_info.append((path_img, int(label)))\n",
    "        return data_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#宏定义一些数据，如epoch数，batchsize等\n",
    "MAX_EPOCH=45\n",
    "BATCH_SIZE=128\n",
    "# LR=0.0001\n",
    "LR=0.001\n",
    "log_interval=30\n",
    "val_interval=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================ step 1/5 数据 ============================\n",
    "# split_dir=os.path.join(\"MobileNetV3-master\",\".\",\"data\",\"splitData\")\n",
    "split_dir=r'/home/fanrz/Desktop/torchmob/'\n",
    "train_dir=r'/home/fanrz/Desktop/testlarge/aptos/tra/'\n",
    "valid_dir=os.path.join(split_dir,\"val\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#对训练集所需要做的预处理\n",
    "train_transform=transforms.Compose([\n",
    "    transforms.Resize((224,224)),\n",
    "    transforms.ToTensor(),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#对验证集所需要做的预处理\n",
    "valid_transform=transforms.Compose([\n",
    "    transforms.Resize((224,224)),\n",
    "    transforms.ToTensor(),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建MyDataset实例\n",
    "train_data=flowerDataset(data_dir=train_dir,transform=train_transform)\n",
    "valid_data=flowerDataset(data_dir=valid_dir,transform=valid_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3295\n"
     ]
    }
   ],
   "source": [
    "print(len(train_data.data_info))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建DataLoader\n",
    "# 训练集数据最好打乱\n",
    "# DataLoader的实质就是把数据集加上一个索引号，再返回\n",
    "train_loader=DataLoader(dataset=train_data,\n",
    "                        batch_size=BATCH_SIZE,\n",
    "                        shuffle=True,\n",
    "                        drop_last=True\n",
    "                       )\n",
    "valid_loader=DataLoader(dataset=valid_data,batch_size=BATCH_SIZE,drop_last=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================ step 2/5 模型 ============================\n",
    "if torch.cuda.is_available():\n",
    "    model=nn.DataParallel(model)\n",
    "    model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================ step 3/5 损失函数 ============================\n",
    "criterion=nn.CrossEntropyLoss()\n",
    "# ============================ step 4/5 优化器 ============================\n",
    "optimizer=optim.Adam(model.parameters(),lr=LR, betas=(0.9, 0.99))# 选择优化器\n",
    "\n",
    "# optimizer2=optim.Adam(model.parameters(),lr=LR, betas=(0.9, 0.99))# 选择优化器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gs(output,label):\n",
    "    s=0.0\n",
    "    for i,n in enumerate(label):\n",
    "        s+=output[i][n]\n",
    "    return s/(i+1)\n",
    "def gw(weights):\n",
    "    s=0\n",
    "    for i in weights:\n",
    "        if(i>0):\n",
    "            s+=i\n",
    "    return s\n",
    "def cw(grad_block,batch):\n",
    "    count_weight_list=[]\n",
    "    i=0\n",
    "    while(i<batch):\n",
    "        grads_val=grad_block[0][i].cpu().data.numpy().squeeze()\n",
    "        grads=grads_val.reshape(grads_val.shape[0],-1)\n",
    "        weights=np.mean(grads,axis=1)\n",
    "        w=gw(weights)\n",
    "        count_weight_list.append(w)\n",
    "        grads=0\n",
    "        grads_val=0\n",
    "        i+=1\n",
    "    return np.mean(count_weight_list)\n",
    "def count_final(grad_block,grad_block2,grad_block3,batch_size):\n",
    "    score_last=cw(grad_block,batch_size)\n",
    "    score_mid=cw(grad_block2,batch_size)\n",
    "    score_first=cw(grad_block3,batch_size)\n",
    "    w_last=score_last/(score_last+score_first+score_mid)\n",
    "    w_mid=score_mid/(score_last+score_first+score_mid)\n",
    "    w_first=score_first/(score_last+score_first+score_mid)\n",
    "    return w_first,w_mid,w_last"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/torch/nn/modules/upsampling.py:129: UserWarning: nn.Upsample is deprecated. Use nn.functional.interpolate instead.\n",
      "  warnings.warn(\"nn.{} is deprecated. Use nn.functional.interpolate instead.\".format(self.name))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "============================================\n",
      "准确率由： 0.0 上升至： 0.6546875 已更新并保存权值为weights/best.pkl\n",
      "第1个epoch的识别准确率为：65%\n",
      "============================================\n",
      "准确率由： 0.6546875 上升至： 0.711875 已更新并保存权值为weights/best.pkl\n",
      "第2个epoch的识别准确率为：71%\n",
      "============================================\n",
      "准确率由： 0.711875 上升至： 0.72875 已更新并保存权值为weights/best.pkl\n",
      "第3个epoch的识别准确率为：72%\n",
      "============================================\n",
      "准确率由： 0.72875 上升至： 0.741875 已更新并保存权值为weights/best.pkl\n",
      "第4个epoch的识别准确率为：74%\n",
      "============================================\n",
      "准确率由： 0.741875 上升至： 0.7571875 已更新并保存权值为weights/best.pkl\n",
      "第5个epoch的识别准确率为：75%\n",
      "============================================\n",
      "准确率由： 0.7571875 上升至： 0.7828125 已更新并保存权值为weights/best.pkl\n",
      "第6个epoch的识别准确率为：78%\n",
      "============================================\n",
      "准确率由： 0.7828125 上升至： 0.8 已更新并保存权值为weights/best.pkl\n",
      "第7个epoch的识别准确率为：80%\n",
      "============================================\n",
      "准确率由： 0.8 上升至： 0.8059375 已更新并保存权值为weights/best.pkl\n",
      "第8个epoch的识别准确率为：80%\n",
      "============================================\n",
      "准确率由： 0.8059375 上升至： 0.83625 已更新并保存权值为weights/best.pkl\n",
      "第9个epoch的识别准确率为：83%\n",
      "============================================\n",
      "准确率由： 0.83625 上升至： 0.838125 已更新并保存权值为weights/best.pkl\n",
      "第10个epoch的识别准确率为：83%\n",
      "============================================\n",
      "准确率由： 0.838125 上升至： 0.86125 已更新并保存权值为weights/best.pkl\n",
      "第11个epoch的识别准确率为：86%\n",
      "============================================\n",
      "准确率由： 0.86125 上升至： 0.8665625 已更新并保存权值为weights/best.pkl\n",
      "第12个epoch的识别准确率为：86%\n",
      "============================================\n",
      "准确率由： 0.8665625 上升至： 0.871875 已更新并保存权值为weights/best.pkl\n",
      "第13个epoch的识别准确率为：87%\n",
      "============================================\n",
      "准确率由： 0.871875 上升至： 0.8784375 已更新并保存权值为weights/best.pkl\n",
      "第14个epoch的识别准确率为：87%\n",
      "============================================\n",
      "准确率由： 0.8784375 上升至： 0.90875 已更新并保存权值为weights/best.pkl\n",
      "第15个epoch的识别准确率为：90%\n",
      "============================================\n",
      "第16个epoch的识别准确率为：90%\n",
      "============================================\n",
      "准确率由： 0.90875 上升至： 0.91 已更新并保存权值为weights/best.pkl\n",
      "第17个epoch的识别准确率为：91%\n",
      "============================================\n",
      "准确率由： 0.91 上升至： 0.92 已更新并保存权值为weights/best.pkl\n",
      "第18个epoch的识别准确率为：92%\n",
      "============================================\n",
      "准确率由： 0.92 上升至： 0.9325 已更新并保存权值为weights/best.pkl\n",
      "第19个epoch的识别准确率为：93%\n",
      "============================================\n",
      "准确率由： 0.9325 上升至： 0.9365625 已更新并保存权值为weights/best.pkl\n",
      "第20个epoch的识别准确率为：93%\n",
      "============================================\n",
      "准确率由： 0.9365625 上升至： 0.9390625 已更新并保存权值为weights/best.pkl\n",
      "第21个epoch的识别准确率为：93%\n",
      "============================================\n",
      "准确率由： 0.9390625 上升至： 0.94125 已更新并保存权值为weights/best.pkl\n",
      "第22个epoch的识别准确率为：94%\n",
      "============================================\n",
      "准确率由： 0.94125 上升至： 0.94375 已更新并保存权值为weights/best.pkl\n",
      "第23个epoch的识别准确率为：94%\n",
      "============================================\n",
      "第24个epoch的识别准确率为：93%\n",
      "============================================\n",
      "准确率由： 0.94375 上升至： 0.9603125 已更新并保存权值为weights/best.pkl\n",
      "第25个epoch的识别准确率为：96%\n",
      "============================================\n",
      "准确率由： 0.9603125 上升至： 0.9609375 已更新并保存权值为weights/best.pkl\n",
      "第26个epoch的识别准确率为：96%\n",
      "============================================\n",
      "准确率由： 0.9609375 上升至： 0.9646875 已更新并保存权值为weights/best.pkl\n",
      "第27个epoch的识别准确率为：96%\n",
      "============================================\n",
      "第28个epoch的识别准确率为：95%\n",
      "============================================\n",
      "第29个epoch的识别准确率为：95%\n",
      "============================================\n",
      "第30个epoch的识别准确率为：96%\n",
      "============================================\n",
      "第31个epoch的识别准确率为：95%\n",
      "============================================\n",
      "第32个epoch的识别准确率为：95%\n"
     ]
    }
   ],
   "source": [
    "# ============================ step 5/5 训练 ============================\n",
    "# 记录每一次的数据，方便绘图\n",
    "train_curve=list()\n",
    "valid_curve=list()\n",
    "model.train()\n",
    "accurancy_global=0.0\n",
    "# model.layers[-1].register_forward_hook(farward_hook)\n",
    "# model.layers[-1].register_backward_hook(backward_hook)\n",
    "# model.layers[-5].register_backward_hook(backward2_hook)\n",
    "# model.layers[-11].register_backward_hook(backward3_hook)\n",
    "for epoch in range(MAX_EPOCH):\n",
    "    loss_mean=0.\n",
    "    correct=0.\n",
    "    total=0.\n",
    "    running_loss = 0.0\n",
    "\n",
    "    for i,data in enumerate(train_loader):\n",
    "        \n",
    "        img,label=data\n",
    "        img = Variable(img)\n",
    "        label = Variable(label)\n",
    "        if torch.cuda.is_available():\n",
    "            img=img.cuda()\n",
    "            label=label.cuda()\n",
    "        \n",
    "        out=model(img)\n",
    "        \n",
    "#         optimizer2.zero_grad()\n",
    "#         class_loss=gs(out,label)\n",
    "#         class_loss.backward(retain_graph=True)\n",
    "        \n",
    "#         w_f,w_m,w_l=count_final(grad_block,grad_block2,grad_block3,BATCH_SIZE)\n",
    "        \n",
    "            \n",
    "#         grad_block=[]\n",
    "#         grad_block2=[]\n",
    "#         grad_block3=[]\n",
    "        \n",
    "        optimizer.zero_grad()  # 归0梯度\n",
    "        ###########################################\n",
    "#         print_loss=loss.data.item()\n",
    "        ###########################################\n",
    "        \n",
    "        loss=criterion(out,label)#得到损失函数\n",
    "        \n",
    "#         class_loss.backward(retain_graph=True)\n",
    "        ###########################################\n",
    "        loss.backward(retain_graph=False)#反向传播\n",
    "        ###########################################\n",
    "        optimizer.step()#优化\n",
    "        \n",
    "        \n",
    "        if (i+1)%log_interval==0:\n",
    "            print('epoch:{},loss:{:.4f}'.format(epoch+1,loss.data.item()))\n",
    "        _, predicted = torch.max(out.data, 1)\n",
    "        total += label.size(0)\n",
    "#         if (i+1)%log_interval==0:\n",
    "#             print(\"============================================\")\n",
    "#             print(\"源数据标签：\",label)\n",
    "#             print(\"============================================\")\n",
    "#             print(\"预测结果：\",predicted)\n",
    "#             print(\"相等的结果为：\",predicted == label)\n",
    "        correct += (predicted == label).sum()\n",
    "        if (i+1)%log_interval==0:\n",
    "            print(correct.item() / total)\n",
    "#         print(correct.item())\n",
    "\n",
    "#     fi=epoch\n",
    "#     if (w_l>0.7)and(fi==0):\n",
    "#         fi+=1\n",
    "#         model.weights_try=w_f\n",
    "#         model.weights2_try=w_m\n",
    "#         model.weights3_try=w_l\n",
    "#         print(w_f,w_m,w_l)\n",
    "#         print('固定参数')\n",
    "#         print(fi)\n",
    "#     model.weights_try=w_f\n",
    "#     model.weights2_try=w_m\n",
    "#     model.weights3_try=w_l\n",
    "    print(\"============================================\")\n",
    "#     print(w_f,w_m,w_l)\n",
    "    accurancy=correct.item() / total\n",
    "    if accurancy>accurancy_global:\n",
    "        torch.save(model.state_dict(), './weights/bestcatapot.pkl')\n",
    "        print(\"准确率由：\", accurancy_global, \"上升至：\", accurancy, \"已更新并保存权值为weights/best.pkl\")\n",
    "        accurancy_global=accurancy\n",
    "    print('第%d个epoch的识别准确率为：%d%%' % (epoch + 1, 100*accurancy))\n",
    "# torch.save(model.state_dict(), './weights/last322.pkl')\n",
    "print(\"训练完毕，权重已保存为：weights/last.pkl\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
